package com.atguigu.gmall.realtime.app.dwd.log;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.atguigu.gmall.realtime.util.DateFormatUtil;
import com.atguigu.gmall.realtime.util.KafkaUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.state.hashmap.HashMapStateBackend;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;

public class DwdTrafficBaseLogSplit {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(4);
      //开启检查点
        env.enableCheckpointing(5000L);
        //设置检查点过期时间
        env.getCheckpointConfig().setCheckpointTimeout(60000L);
        //设置job取消之后检查点是否保留
        env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
        //设置两个检查点的最小时间间隔
        env.getCheckpointConfig().setMinPauseBetweenCheckpoints(2000L);
        //设置重启策略
        env.setRestartStrategy(RestartStrategies.failureRateRestart(3, Time.days(30), Time.seconds(3)));
        //设置状态后端
        env.setStateBackend(new HashMapStateBackend());
        env.getCheckpointConfig().setCheckpointStorage("hdfs://hadoop100:8020/edu/ck");
        //获取kafka消费者
        String topic = "topic_log";
        String groupId = "topic_consumer";
        FlinkKafkaConsumer<String> kafkaConsumer = KafkaUtil.getKafkaConsumer(topic, groupId);
        DataStreamSource<String> stringDS = env.addSource(kafkaConsumer);

        OutputTag<String> outputTag = new OutputTag<String>("dirtyTag") {
        };

        SingleOutputStreamOperator<JSONObject> jsonDs = stringDS.process(
                new ProcessFunction<String, JSONObject>() {

                    @Override
                    public void processElement(String jsonStr, Context context, Collector<JSONObject> collector) throws Exception {
                        try {
                            JSONObject jsonObject = JSON.parseObject(jsonStr);
                            collector.collect(jsonObject);
                        } catch (Exception e) {
                            System.out.println("脏数据");
                            context.output(outputTag, jsonStr);
                        }
                    }
                }
        );
        //将测输出流数据写入到专门的kafka主题中
        DataStream<String> sideOutput = jsonDs.getSideOutput(outputTag);
        sideOutput.addSink(KafkaUtil.getKafkaProduce("dirty_data"));

        KeyedStream<JSONObject, String> keyedStream = jsonDs.keyBy(jsonObj -> jsonObj.getJSONObject("common").getString("mid"));
        SingleOutputStreamOperator<JSONObject> fixDs = keyedStream.map(new RichMapFunction<JSONObject, JSONObject>() {
            private ValueState<String> lastVisitDateValue;

            @Override
            public void open(Configuration parameters) throws Exception {
                lastVisitDateValue = getRuntimeContext().getState(
                        new ValueStateDescriptor<String>(
                                "curDateValue",
                                Types.STRING
                        )
                );
            }

            @Override
            public JSONObject map(JSONObject jsonObject) throws Exception {
                String isNew = jsonObject.getJSONObject("common").getString("is_new");
                Long ts = jsonObject.getLong("ts");
                String lastVisitDate = lastVisitDateValue.value();
                String curVisitDate = DateFormatUtil.toDate(ts);
                if (isNew.equals("1")) {
                    if (StringUtils.isEmpty(lastVisitDate)) {
                        lastVisitDateValue.update(curVisitDate);
                    } else {
                        if (!lastVisitDate.equals(curVisitDate)) {
                            isNew = "0";
                            jsonObject.getJSONObject("common").put("is_new", isNew);
                        }
                    }
                } else {
                    if (lastVisitDate == null) {
                        String yesterDay = DateFormatUtil.toDate(ts - 24 * 60 * 60 * 1000);
                        lastVisitDateValue.update(yesterDay);
                    }
                }
                return jsonObject;
            }
        });

        //分流
        OutputTag<String> startTag = new OutputTag<String>("startTag"){};
        OutputTag<String> errTag = new OutputTag<String>("errTag"){};
        OutputTag<String> displayTag = new OutputTag<String>("displayTag"){};
        OutputTag<String> actionTag = new OutputTag<String>("actionTag"){};

        SingleOutputStreamOperator<String> pageDs = fixDs.process(new ProcessFunction<JSONObject, String>() {
            @Override
            public void processElement(JSONObject jsonObject, Context context, Collector<String> collector) throws Exception {
                JSONObject err = jsonObject.getJSONObject("err");
                //判断是否为错误日志
                if (err != null) {
                    context.output(errTag, err.toJSONString());
                    jsonObject.remove("err");
                }
                //判断是否为启动日志
                JSONObject start = jsonObject.getJSONObject("start");
                if (start != null && start.size() > 0) {
                    context.output(startTag, start.toJSONString());
                } else {
                    JSONObject page = jsonObject.getJSONObject("page");
                    JSONObject common = jsonObject.getJSONObject("common");
                    Long ts = jsonObject.getLong("ts");
                    //判断页面上是否存在曝光
                    JSONArray displays = jsonObject.getJSONArray("displays");
                    if (displays != null && displays.size() > 0) {
                        for (int i = 0; i < displays.size(); i++) {
                            JSONObject displaysJSONObject = displays.getJSONObject(i);
                            JSONObject newJsonObj = new JSONObject();
                            newJsonObj.put("page", page);
                            newJsonObj.put("common", common);
                            newJsonObj.put("display", displaysJSONObject);
                            newJsonObj.put("ts", ts);
                            context.output(displayTag, newJsonObj.toJSONString());
                        }
                    }
                    //判断页面上是否存在动作
                    JSONArray actionArray = jsonObject.getJSONArray("actions");
                    if (actionArray != null && actionArray.size() > 0) {
                        for (int i = 0; i < actionArray.size(); i++) {
                            JSONObject actionJsonObj = actionArray.getJSONObject(i);
                            JSONObject newActionJson = new JSONObject();
                            newActionJson.put("page", page);
                            newActionJson.put("common", common);
                            newActionJson.put("action", actionJsonObj);
                            context.output(actionTag, newActionJson.toJSONString());
                        }
                    }
                    //页面日志输出到主流中
                    jsonObject.remove("actions");
                    jsonObject.remove("displays");
                    collector.collect(jsonObject.toJSONString());
                }
            }
        });
        //将不同数据输出到不同的kafka主题中
        DataStream<String> startDs = pageDs.getSideOutput(startTag);
        DataStream<String> errDs = pageDs.getSideOutput(errTag);
        DataStream<String> displayDs = pageDs.getSideOutput(displayTag);
        DataStream<String> actionDs = pageDs.getSideOutput(actionTag);

        pageDs.print(">>>>");
        startDs.print("@@@@");
        errDs.print("$$$$$");
        displayDs.print("#####");
        actionDs.print("****");

        startDs.addSink(KafkaUtil.getKafkaProduce("dwd_traffic_start_log"));
        errDs.addSink(KafkaUtil.getKafkaProduce("dwd_traffic_err_log"));
        pageDs.addSink(KafkaUtil.getKafkaProduce("dwd_traffic_page_log"));
        actionDs.addSink(KafkaUtil.getKafkaProduce("dwd_traffic_action_log"));
        displayDs.addSink(KafkaUtil.getKafkaProduce("dwd_traffic_display_log"));

        env.execute();
    }
}
